{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([24.       ,  0.       , 24.       , 32.       , 24.       ,\n",
       "       36.       ,  7.1824338])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pong-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (7,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [ 24.           0.          24.          32.          24.\n",
      "  36.         -10.07757721]\n",
      "action= 0\n",
      "next_state= [ 23.424       -0.576       23.04        33.2         23.66408076\n",
      "  36.         -10.07757721]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARcAAADdCAYAAACR1zayAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAR4UlEQVR4nO3dbWxT5fsH8O9G1zPG1o4N17JshSUuToIoDBgVExLXSJQoD4vhhUZUItkouMEbshdgTNQu8kLFIPgQwURkZiagI1FcBsyQlMFKpjw5IJCscWsnMT0dD3vIev3f/P4nFthGt962G99PciXuPvc55+Ke/ebsnG5NEREBEVGcpSa6ASKanBguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkhLJw2b17N2bPno309HSUlZXh9OnTqk5FREkoRcXvFn3//fd47bXXsHfvXpSVleHjjz9GQ0MDOjo6kJeXN+K+kUgEXV1dyMrKQkpKSrxbI6JxEhH09vYiPz8fqakjXJ+IAosXLxa32218PTQ0JPn5+eLxeEbd1+/3CwAWi5Xk5ff7R3wtx/3HooGBAfh8PrhcLmMsNTUVLpcLXq/3nvn9/f0Ih8NGCX9Jm2hCyMrKGnF73MPlxo0bGBoags1mixq32WwIBAL3zPd4PLBarUY5HI54t0RECox22yLhT4tqa2uh67pRfr8/0S0RURyY4n3AGTNmYMqUKQgGg1HjwWAQdrv9nvmapkHTtHi3QUQJFvcrF7PZjNLSUjQ3NxtjkUgEzc3NcDqd8T4dESWr8T4Zup/6+nrRNE32798vFy9elA0bNkh2drYEAoFR99V1PeF3wVks1uil6/qIr+W4/1gEAGvXrsXff/+NHTt2IBAI4KmnnsIvv/xyz01eIpq8lLyJbjzC4TCsVmui2yCiUei6DovFMuz2hD8tIqLJieFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSIuZw+e233/Diiy8iPz8fKSkpOHz4cNR2EcGOHTswc+ZMTJ06FS6XC1euXIlXv0Q0QcQcLrdu3cKTTz6J3bt333f7hx9+iF27dmHv3r1obW3FtGnTsHz5cvT19Y27WSKaQEb8mPpRAJBDhw4ZX0ciEbHb7bJz505jLBQKiaZpcvDgwQc6pq7rAoDFYiV56bo+4ms5rvdcrl+/jkAgAJfLZYxZrVaUlZXB6/Xed5/+/n6Ew+GoIqKJL67hEggEAAA2my1q3GazGdvu5vF4YLVajSosLIxnS0SUIAl/WlRbWwtd143y+/2JbomI4iCu4WK32wEAwWAwajwYDBrb7qZpGiwWS1QR0cQX13ApKiqC3W5Hc3OzMRYOh9Ha2gqn0xnPUxFRkjPFusPNmzdx9epV4+vr16+jvb0dOTk5cDgcqKmpwXvvvYfi4mIUFRVh+/btyM/Px6pVq+LZNxElu1gfPx8/fvy+j6XWrVtnPI7evn272Gw20TRNysvLpaOj44GPz0fRLNbEqNEeRaeIiCCJhMNhWK3WRLdBRKPQdX3Ee6QJf1pERJMTw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlIgpXDweDxYtWoSsrCzk5eVh1apV6OjoiJrT19cHt9uN3NxcZGZmoqKiAsFgMK5NE1HyiylcWlpa4Ha7cerUKTQ1NWFwcBDPPfccbt26ZczZsmULGhsb0dDQgJaWFnR1dWHNmjVxb3w4ImIUEQ3v368VJa+XET+mfhQ9PT0CQFpaWkREJBQKSVpamjQ0NBhzLl26JADE6/Xe9xh9fX2i67pRfr9fAIy5/m08x2GxJnvdLdb9dV0fMR/Gdc9F13UAQE5ODgDA5/NhcHAQLpfLmFNSUgKHwwGv13vfY3g8HlitVqMKCwvH0xIRJYkxh0skEkFNTQ2WLl2KuXPnAgACgQDMZjOys7Oj5tpsNgQCgfsep7a2FrquG+X3+8faEhElEdNYd3S73Th//jxOnjw5rgY0TYOmaeM6BhElnzFduWzatAlHjhzB8ePHUVBQYIzb7XYMDAwgFApFzQ8Gg7Db7eNqlIgmmFhu4EYiEXG73ZKfny+XL1++Z/v/39D94YcfjLE///xTgOFv6N5N1/W43aQaz3FYrMled4t1/9Fu6MYULlVVVWK1WuXEiRPS3d1t1O3bt405lZWV4nA45NixY9LW1iZOp1OcTucDn4PhMvlqNInu72Gt8X4f4houw51k3759xpw7d+7Ixo0bZfr06ZKRkSGrV6+W7u7uBz4Hw2Xy1Vj/v2L9t9+XWPcfLVxS/nfQpBEOh2G1Wse8/7//OSkpKfFoicZptP/F+H1KjLu/L7F+H3Rdh8ViGXY7f7eIiJRguBCREgwXIlKC4UJESoz5HbpED4o3bB9OvHIhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpMSke/s/32pO9GBUv1Z45UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCRErEFC579uzBvHnzYLFYYLFY4HQ68fPPPxvb+/r64Ha7kZubi8zMTFRUVCAYDMa9aSJKfjGFS0FBAerq6uDz+dDW1oZnn30WK1euxIULFwAAW7ZsQWNjIxoaGtDS0oKuri6sWbNGSeNElOQe+BPihzF9+nT56quvJBQKSVpamjQ0NBjbLl26JADE6/U+8PHG+0H0LBbrv6nRPoh+zPdchoaGUF9fj1u3bsHpdMLn82FwcBAul8uYU1JSAofDAa/XO+xx+vv7EQ6Ho4qIJr6Yw+XcuXPIzMyEpmmorKzEoUOHMGfOHAQCAZjNZmRnZ0fNt9lsCAQCwx7P4/HAarUaVVhYGPM/goiST8zh8thjj6G9vR2tra2oqqrCunXrcPHixTE3UFtbC13XjfL7/WM+FhElj5j/novZbMajjz4KACgtLcWZM2fwySefYO3atRgYGEAoFIq6egkGg7Db7cMeT9M0aJoWe+dElNTG/T6XSCSC/v5+lJaWIi0tDc3Nzca2jo4OdHZ2wul0jvc0RDTBxHTlUltbi+effx4OhwO9vb347rvvcOLECRw9ehRWqxXr16/H1q1bkZOTA4vFgs2bN8PpdGLJkiWq+ieiZBXLY+c333xTZs2aJWazWR555BEpLy+XX3/91dh+584d2bhxo0yfPl0yMjJk9erV0t3dHcsp+CiaxZogNdqj6BQRESSRcDgMq9Wa6DaIaBS6rsNisQy7nb9bRERKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKjCtc6urqkJKSgpqaGmOsr68Pbrcbubm5yMzMREVFBYLB4Hj7JKIJZszhcubMGXz++eeYN29e1PiWLVvQ2NiIhoYGtLS0oKurC2vWrBl3o0Q0wcT0KfH/09vbK8XFxdLU1CTLli2T6upqEREJhUKSlpYmDQ0NxtxLly4JAPF6vfwgehZrEtVoH0Q/pisXt9uNFStWwOVyRY37fD4MDg5GjZeUlMDhcMDr9d73WP39/QiHw1FFRBOfKdYd6uvrcfbsWZw5c+aebYFAAGazGdnZ2VHjNpsNgUDgvsfzeDx49913Y22DiJJcTFcufr8f1dXVOHDgANLT0+PSQG1tLXRdN8rv98fluESUWDGFi8/nQ09PDxYsWACTyQSTyYSWlhbs2rULJpMJNpsNAwMDCIVCUfsFg0HY7fb7HlPTNFgslqgiookvph+LysvLce7cuaixN954AyUlJdi2bRsKCwuRlpaG5uZmVFRUAAA6OjrQ2dkJp9MZv66JKOnFFC5ZWVmYO3du1Ni0adOQm5trjK9fvx5bt25FTk4OLBYLNm/eDKfTiSVLlsSvayJKejHf0B3NRx99hNTUVFRUVKC/vx/Lly/HZ599Fu/TEFGSSxERSXQT/xYOh2G1WhPdBhGNQtf1Ee+R8neLiEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiWSLlyS7E/6EtEwRnutJl249Pb2JroFInoAo71Wk+6v/0ciEXR1dUFE4HA44Pf7+SmMwwiHwygsLOQaDYPrM7qxrJGIoLe3F/n5+UhNHf76JO6fWzReqampKCgoQDgcBgB+xOsD4BqNjOszuljX6EE+/ifpfiwiosmB4UJESiRtuGiahnfeeQeapiW6laTFNRoZ12d0Ktco6W7oEtHkkLRXLkQ0sTFciEgJhgsRKcFwISIlGC5EpETShsvu3bsxe/ZspKeno6ysDKdPn050Swnh8XiwaNEiZGVlIS8vD6tWrUJHR0fUnL6+PrjdbuTm5iIzMxMVFRUIBoMJ6jix6urqkJKSgpqaGmOM6wP89ddfePXVV5Gbm4upU6fiiSeeQFtbm7FdRLBjxw7MnDkTU6dOhcvlwpUrV8Z3UklC9fX1Yjab5euvv5YLFy7IW2+9JdnZ2RIMBhPd2n9u+fLlsm/fPjl//ry0t7fLCy+8IA6HQ27evGnMqayslMLCQmlubpa2tjZZsmSJPP300wnsOjFOnz4ts2fPlnnz5kl1dbUx/rCvzz///COzZs2S119/XVpbW+XatWty9OhRuXr1qjGnrq5OrFarHD58WH7//Xd56aWXpKioSO7cuTPm8yZluCxevFjcbrfx9dDQkOTn54vH40lgV8mhp6dHAEhLS4uIiIRCIUlLS5OGhgZjzqVLlwSAeL3eRLX5n+vt7ZXi4mJpamqSZcuWGeHC9RHZtm2bPPPMM8Nuj0QiYrfbZefOncZYKBQSTdPk4MGDYz5v0v1YNDAwAJ/PB5fLZYylpqbC5XLB6/UmsLPkoOs6ACAnJwcA4PP5MDg4GLVeJSUlcDgcD9V6ud1urFixImodAK4PAPz0009YuHAhXn75ZeTl5WH+/Pn48ssvje3Xr19HIBCIWiOr1YqysrJxrVHShcuNGzcwNDQEm80WNW6z2RAIBBLUVXKIRCKoqanB0qVLMXfuXABAIBCA2WxGdnZ21NyHab3q6+tx9uxZeDyee7ZxfYBr165hz549KC4uxtGjR1FVVYW3334b33zzDQAY6xDv11zS/ckFGp7b7cb58+dx8uTJRLeSNPx+P6qrq9HU1IT09PREt5OUIpEIFi5ciA8++AAAMH/+fJw/fx579+7FunXrlJ036a5cZsyYgSlTptxzNz8YDMJutyeoq8TbtGkTjhw5guPHj6OgoMAYt9vtGBgYQCgUipr/sKyXz+dDT08PFixYAJPJBJPJhJaWFuzatQsmkwk2m+2hXh8AmDlzJubMmRM19vjjj6OzsxMAjHWI92su6cLFbDajtLQUzc3NxlgkEkFzczOcTmcCO0sMEcGmTZtw6NAhHDt2DEVFRVHbS0tLkZaWFrVeHR0d6OzsfCjWq7y8HOfOnUN7e7tRCxcuxCuvvGL898O8PgCwdOnSe96+cPnyZcyaNQsAUFRUBLvdHrVG4XAYra2t41ujMd8KVqi+vl40TZP9+/fLxYsXZcOGDZKdnS2BQCDRrf3nqqqqxGq1yokTJ6S7u9uo27dvG3MqKyvF4XDIsWPHpK2tTZxOpzidzgR2nVj/flokwvU5ffq0mEwmef/99+XKlSty4MABycjIkG+//daYU1dXJ9nZ2fLjjz/KH3/8IStXrpycj6JFRD799FNxOBxiNptl8eLFcurUqUS3lBAA7lv79u0z5ty5c0c2btwo06dPl4yMDFm9erV0d3cnrukEuztcuD4ijY2NMnfuXNE0TUpKSuSLL76I2h6JRGT79u1is9lE0zQpLy+Xjo6OcZ2Tf8+FiJRIunsuRDQ5MFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKfF/j5QCXksZ6qcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7fdeb805f3d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-10.65, 3.7319565913874184)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/3.PPO.Pong')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.2, 0.4000000000000001)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/3.PPO.Pong')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARcAAADdCAYAAACR1zayAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAARuUlEQVR4nO3da2xT5R8H8O9G1zPG1u6Ca1lYYYmLkyAKA0bFhMQ1EiXKZTG80IhKJBsFN3hD9gKMidpFXqgYBC8RTERmZgI6EsVlwAxJ2VgJys0BgWSNWzuJ6em47JL193/j/8TKLnTrYzv4fpJfwp7z9JwfT9JvTs9Zd1JEREBEFGepiW6AiO5PDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlJCWbjs3r0bs2fPRnp6OsrKytDW1qbqUESUhFJUfLfo22+/xSuvvIK9e/eirKwMH374IRoaGtDR0YH8/PxRXxuJRNDV1YWsrCykpKTEuzUimiARQW9vLwoKCpCaOsr5iSiwePFicbvdxs9DQ0NSUFAgHo9nzNf6/X4BwGKxkrz8fv+o7+W4fywaGBiAz+eDy+UyxlJTU+FyueD1eu+a39/fj3A4bJTwS9pEk0JWVtao2+MeLjdu3MDQ0BBsNlvUuM1mQyAQuGu+x+OB1Wo1yuFwxLslIlJgrMsWCb9bVFtbC13XjfL7/YluiYjiwBTvHU6fPh1TpkxBMBiMGg8Gg7Db7XfN1zQNmqbFuw0iSrC4n7mYzWaUlpaiubnZGItEImhubobT6Yz34YgoWU30ztBw6uvrRdM02b9/v1y8eFE2bNgg2dnZEggExnytrusJvwrOYrHGLl3XR30vx/1jEQCsXbsWf/75J3bs2IFAIIAnnngCP/30010XeYno/qXkl+gmIhwOw2q1JroNIhqDruuwWCwjbk/43SIiuj8xXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCRErEHC6//PILnn/+eRQUFCAlJQWHDx+O2i4i2LFjB2bMmIGpU6fC5XLhypUr8eqXiCaJmMPl1q1bePzxx7F79+5ht7///vvYtWsX9u7di9bWVkybNg3Lly9HX1/fhJsloklk1MfUjwGAHDp0yPg5EomI3W6XnTt3GmOhUEg0TZODBw/e0z51XRcALBYryUvX9VHfy3G95nL9+nUEAgG4XC5jzGq1oqysDF6vd9jX9Pf3IxwORxURTX5xDZdAIAAAsNlsUeM2m83Y9m8ejwdWq9WowsLCeLZERAmS8LtFtbW10HXdKL/fn+iWiCgO4houdrsdABAMBqPGg8Ggse3fNE2DxWKJKiKa/OIaLkVFRbDb7WhubjbGwuEwWltb4XQ643koIkpyplhfcPPmTVy9etX4+fr16zh79ixyc3PhcDhQU1ODd955B8XFxSgqKsL27dtRUFCAVatWxbNvIkp2sd5+Pn78+LC3pdatW2fcjt6+fbvYbDbRNE3Ky8ulo6PjnvfPW9Es1uSosW5Fp4iIIImEw2FYrdZEt0FEY9B1fdRrpAm/W0RE9yeGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpEVO4eDweLFq0CFlZWcjPz8eqVavQ0dERNaevrw9utxt5eXnIzMxERUUFgsFgXJsmouQXU7i0tLTA7Xbj1KlTaGpqwuDgIJ555hncunXLmLNlyxY0NjaioaEBLS0t6Orqwpo1a+LeOBEluVEfUz+Gnp4eASAtLS0iIhIKhSQtLU0aGhqMOZcuXRIA4vV6h91HX1+f6LpulN/vFwAsFivJS9f1UfNhQtdcdF0HAOTm5gIAfD4fBgcH4XK5jDklJSVwOBzwer3D7sPj8cBqtRpVWFg4kZaIKEmMO1wikQhqamqwdOlSzJ07FwAQCARgNpuRnZ0dNddmsyEQCAy7n9raWui6bpTf7x9vS0SUREzjfaHb7cb58+dx8uTJCTWgaRo0TZvQPogo+YzrzGXTpk04cuQIjh8/jpkzZxrjdrsdAwMDCIVCUfODwSDsdvuEGiWiSSaWC7iRSETcbrcUFBTI5cuX79r+/wu63333nTH2+++/CzDyBd1/03U94ReqWCzW2DXWBd2YwqWqqkqsVqucOHFCuru7jbp9+7Yxp7KyUhwOhxw7dkza29vF6XSK0+m852MwXFisyVFxDZeRDrJv3z5jzp07d2Tjxo2Sk5MjGRkZsnr1aunu7ma4sFj3WY0VLil/h0bSCIfDsFqtiW6DiMag6zosFsuI2/ndIiJSYty3oolocvv3h5aUlJS47p9nLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEiJ++5W9D9vr8X71hoR3TueuRCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJSYtL/+n+S/QlgIvobz1yISAmGCxEpwXAhIiUYLkSkBMOFiJSIKVz27NmDefPmwWKxwGKxwOl04scffzS29/X1we12Iy8vD5mZmaioqEAwGIx700SU/GIKl5kzZ6Kurg4+nw/t7e14+umnsXLlSly4cAEAsGXLFjQ2NqKhoQEtLS3o6urCmjVrlDT+fykpKVFFRPdG+XsnlgfRDycnJ0e++OILCYVCkpaWJg0NDca2S5cuCQDxer33vL+JPoj+nyayHxaLNXqN9SD6cV9zGRoaQn19PW7dugWn0wmfz4fBwUG4XC5jTklJCRwOB7xe74j76e/vRzgcjioimvxiDpdz584hMzMTmqahsrIShw4dwpw5cxAIBGA2m5GdnR0132azIRAIjLg/j8cDq9VqVGFhYcz/CSJKPjGHyyOPPIKzZ8+itbUVVVVVWLduHS5evDjuBmpra6HrulF+v3/c+yKi5BHzd4vMZjMefvhhAEBpaSlOnz6Njz76CGvXrsXAwABCoVDU2UswGITdbh9xf5qmQdO02DsnoqQ24d9ziUQi6O/vR2lpKdLS0tDc3Gxs6+joQGdnJ5xO50QPQ0STTExnLrW1tXj22WfhcDjQ29uLb775BidOnMDRo0dhtVqxfv16bN26Fbm5ubBYLNi8eTOcTieWLFmiqn8iSlax3HZ+/fXXZdasWWI2m+Whhx6S8vJy+fnnn43td+7ckY0bN0pOTo5kZGTI6tWrpbu7O5ZDTPhWNIvF+m9qrFvRKSLJ9QdRwuEwrFZrotsgojHoug6LxTLidn63iIiUYLgQkRIMFyJSguFCREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUmFC41NXVISUlBTU1NcZYX18f3G438vLykJmZiYqKCgSDwYn2SUSTzLjD5fTp0/j0008xb968qPEtW7agsbERDQ0NaGlpQVdXF9asWTPhRolokonpKfF/6+3tleLiYmlqapJly5ZJdXW1iIiEQiFJS0uThoYGY+6lS5cEgHi9Xj6InsW6j2qsB9GP68zF7XZjxYoVcLlcUeM+nw+Dg4NR4yUlJXA4HPB6vcPuq7+/H+FwOKqIaPIzxfqC+vp6nDlzBqdPn75rWyAQgNlsRnZ2dtS4zWZDIBAYdn8ejwdvv/12rG0QUZKL6czF7/ejuroaBw4cQHp6elwaqK2tha7rRvn9/rjsl4gSK6Zw8fl86OnpwYIFC2AymWAymdDS0oJdu3bBZDLBZrNhYGAAoVAo6nXBYBB2u33YfWqaBovFElVENPnF9LGovLwc586dixp77bXXUFJSgm3btqGwsBBpaWlobm5GRUUFAKCjowOdnZ1wOp3x65qIkl5M4ZKVlYW5c+dGjU2bNg15eXnG+Pr167F161bk5ubCYrFg8+bNcDqdWLJkSfy6JqKkF/MF3bF88MEHSE1NRUVFBfr7+7F8+XJ88skn8T4MESW5FBGRRDfxT+FwGFarNdFtENEYdF0f9Ropv1tEREowXIhICYYLESnBcCEiJRguRKQEw4WIlGC4EJESDBciUoLhQkRKMFyISAmGCxEpwXAhIiUYLkSkBMOFiJRguBCREgwXIlKC4UJESjBciEgJhgsRKZF04ZJkf9KXiEYw1ns16cKlt7c30S0Q0T0Y672adH/9PxKJoKurCyICh8MBv9/PpzCOIBwOo7CwkGs0Aq7P2MazRiKC3t5eFBQUIDV15POTuD+3aKJSU1Mxc+ZMhMNhAOAjXu8B12h0XJ+xxbpG9/L4n6T7WERE9weGCxEpkbThomka3nrrLWialuhWkhbXaHRcn7GpXKOku6BLRPeHpD1zIaLJjeFCREowXIhICYYLESnBcCEiJZI2XHbv3o3Zs2cjPT0dZWVlaGtrS3RLCeHxeLBo0SJkZWUhPz8fq1atQkdHR9Scvr4+uN1u5OXlITMzExUVFQgGgwnqOLHq6uqQkpKCmpoaY4zrA/zxxx94+eWXkZeXh6lTp+Kxxx5De3u7sV1EsGPHDsyYMQNTp06Fy+XClStXJnZQSUL19fViNpvlyy+/lAsXLsgbb7wh2dnZEgwGE93af2758uWyb98+OX/+vJw9e1aee+45cTgccvPmTWNOZWWlFBYWSnNzs7S3t8uSJUvkySefTGDXidHW1iazZ8+WefPmSXV1tTH+oK/PX3/9JbNmzZJXX31VWltb5dq1a3L06FG5evWqMaeurk6sVqscPnxYfv31V3nhhRekqKhI7ty5M+7jJmW4LF68WNxut/Hz0NCQFBQUiMfjSWBXyaGnp0cASEtLi4iIhEIhSUtLk4aGBmPOpUuXBIB4vd5Etfmf6+3tleLiYmlqapJly5YZ4cL1Edm2bZs89dRTI26PRCJit9tl586dxlgoFBJN0+TgwYPjPm7SfSwaGBiAz+eDy+UyxlJTU+FyueD1ehPYWXLQdR0AkJubCwDw+XwYHByMWq+SkhI4HI4Har3cbjdWrFgRtQ4A1wcAfvjhByxcuBAvvvgi8vPzMX/+fHz++efG9uvXryMQCEStkdVqRVlZ2YTWKOnC5caNGxgaGoLNZosat9lsCAQCCeoqOUQiEdTU1GDp0qWYO3cuACAQCMBsNiM7Oztq7oO0XvX19Thz5gw8Hs9d27g+wLVr17Bnzx4UFxfj6NGjqKqqwptvvomvvvoKAIx1iPd7Lun+5AKNzO124/z58zh58mSiW0kafr8f1dXVaGpqQnp6eqLbSUqRSAQLFy7Ee++9BwCYP38+zp8/j71792LdunXKjpt0Zy7Tp0/HlClT7rqaHwwGYbfbE9RV4m3atAlHjhzB8ePHMXPmTGPcbrdjYGAAoVAoav6Dsl4+nw89PT1YsGABTCYTTCYTWlpasGvXLphMJthstgd6fQBgxowZmDNnTtTYo48+is7OTgAw1iHe77mkCxez2YzS0lI0NzcbY5FIBM3NzXA6nQnsLDFEBJs2bcKhQ4dw7NgxFBUVRW0vLS1FWlpa1Hp1dHSgs7PzgViv8vJynDt3DmfPnjVq4cKFeOmll4x/P8jrAwBLly6969cXLl++jFmzZgEAioqKYLfbo9YoHA6jtbV1Yms07kvBCtXX14umabJ//365ePGibNiwQbKzsyUQCCS6tf9cVVWVWK1WOXHihHR3dxt1+/ZtY05lZaU4HA45duyYtLe3i9PpFKfTmcCuE+ufd4tEuD5tbW1iMpnk3XfflStXrsiBAwckIyNDvv76a2NOXV2dZGdny/fffy+//fabrFy58v68FS0i8vHHH4vD4RCz2SyLFy+WU6dOJbqlhAAwbO3bt8+Yc+fOHdm4caPk5ORIRkaGrF69Wrq7uxPXdIL9O1y4PiKNjY0yd+5c0TRNSkpK5LPPPovaHolEZPv27WKz2UTTNCkvL5eOjo4JHZN/z4WIlEi6ay5EdH9guBCREgwXIlKC4UJESjBciEgJhgsRKcFwISIlGC5EpATDhYiUYLgQkRIMFyJS4n9EVnLtUkivIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 400 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
